/**
 * The contents of this file are subject to the license and copyright detailed in the LICENSE and NOTICE files at the root of the source tree and available online at
 * http://www.askosi.org/license.html
 */
package be.destin.skos.core;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.URL;
import java.util.Date;
import java.util.LinkedList;
import java.util.Properties;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.stream.XMLStreamWriter;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.stax.StAXResult;
import javax.xml.transform.stax.StAXSource;
import javax.xml.transform.stream.StreamSource;

import org.apache.log4j.Logger;
import org.codehaus.jettison.mapped.Configuration;
import org.codehaus.jettison.mapped.MappedNamespaceConvention;
import org.codehaus.jettison.mapped.MappedXMLStreamWriter;

import be.destin.skos.xml.NoDTDInputStream;
import be.destin.skos.xml.NoXmlDeclarationFileOutputStream;
import be.destin.skos.xml.XMLUtil;
import be.destin.util.RunCommand;
import be.destin.util.Util;

public class XmlSource {

	/** log4j category */
	static Logger log = Logger.getLogger(XmlSource.class);
	
	static public JAXBContext jc = null;
	
	public static synchronized JAXBContext getJC() {
		if (jc == null) {
			try {
				jc = JAXBContext.newInstance( "be.destin.skos.core" );
			} catch( JAXBException je ) {
			log.error("get JAXBContext in ASKOSI:", je);
				je.printStackTrace();
				jc = null;
			}
		}
		return jc;
	}

	/**
	 * Serialize an object in XML (ConceptScheme.xsd)
	 */
	public static void marshall (ConceptScheme aScheme, boolean onlyTop, String cacheFileName) {
		OutputStream dataOutputStream = null;
		String dataOutputLocation = "";
		try {
			File cacheFile = new File(cacheFileName);
			dataOutputLocation = cacheFile.getAbsolutePath();
			dataOutputStream = new FileOutputStream(cacheFile);
		} catch( IOException ioe ) {
			dataOutputStream = null;
		}
		if (dataOutputStream != null) {
			marshall(aScheme, onlyTop, new OutputStreamWriter(dataOutputStream));
			RunCommand.closeAble(dataOutputStream, "data output stream " +dataOutputLocation);
		}
	}

	/*TODO: allow to generate XML within a begun stream:
	 * Set the Marshaller.JAXB_FRAGMENT property on the Marshaller to prevent the XML declaration from being written.
	 * marshaller.setProperty(Marshaller.JAXB_FRAGMENT, true); 
	 *  
	*/
	public static synchronized void marshall (ConceptScheme aScheme, boolean onlyTop, Writer writer) {
		JAXBContext jc = null;
		try {
			jc = XmlSource.getJC();
			if (jc != null) {
				Marshaller m = jc.createMarshaller();
				m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
				aScheme.setNoConcepts(onlyTop);
				m.marshal( aScheme, writer );
			}
		} catch( JAXBException je ) {
			log.error("XML generation exception: " + je.toString());
			je.printStackTrace();
		}
		aScheme.setNoConcepts(false);
	}

	public static void marshallConcept (Concept aConcept, Writer writer) {
		JAXBContext jc = null;
		try {
			jc = XmlSource.getJC();
			if (jc != null) {
				Marshaller m = jc.createMarshaller();
				m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
				m.marshal( aConcept, writer );
			}
		} catch( JAXBException je ) {
			log.error("XML generation exception: " + je.toString());
			je.printStackTrace();
		}
	}

	/**
	 * Serialize an object in JSON
	 */
	public static void marshallJSON (ConceptScheme aScheme, boolean onlyTop, String cacheFileName) {
		OutputStream dataOutputStream = null;
		String dataOutputLocation = "";
		try {
			File cacheFile = new File(cacheFileName);
			dataOutputLocation = cacheFile.getAbsolutePath();
			dataOutputStream = new FileOutputStream(cacheFile);
		} catch( IOException ioe ) {
			dataOutputStream = null;
		}
		if (dataOutputStream != null) {
			marshallJSON(aScheme, onlyTop, new OutputStreamWriter(dataOutputStream));
			RunCommand.closeAble(dataOutputStream, "data output stream " +dataOutputLocation);
		}
	}

	public static synchronized void marshallJSON (ConceptScheme aScheme, boolean onlyTop, Writer writer) {
		JAXBContext jc = null;
		try {
			jc = XmlSource.getJC();
			if (jc != null) {

				Configuration config = new Configuration();
				config.setSupressAtAttributes(true);
				MappedNamespaceConvention con = new MappedNamespaceConvention(config);
				XMLStreamWriter xmlStreamWriter = new MappedXMLStreamWriter(con, writer);
				Marshaller marshaller = jc.createMarshaller();
				//m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
				aScheme.setNoConcepts(onlyTop);
				marshaller.marshal(aScheme, xmlStreamWriter);
				xmlStreamWriter.close();
			}
		} catch( XMLStreamException jx ) {
			log.error("JSON Stream exception: " + jx.toString());
			jx.printStackTrace();
		} catch( JAXBException je ) {
			log.error("JSON generation exception: " + je.toString());
			je.printStackTrace();
		}
		aScheme.setNoConcepts(false);
	}

	public static void marshallConceptJSON (Concept aConcept, Writer writer) {
		JAXBContext jc = null;
		try {
			jc = XmlSource.getJC();
			if (jc != null) {

				Configuration config = new Configuration();
				config.setSupressAtAttributes(true);
				MappedNamespaceConvention con = new MappedNamespaceConvention(config);
				XMLStreamWriter xmlStreamWriter = new MappedXMLStreamWriter(con, writer);
				Marshaller marshaller = jc.createMarshaller();
				//m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
				marshaller.marshal(aConcept, xmlStreamWriter); 
				xmlStreamWriter.close();
			}
		} catch( XMLStreamException jx ) {
			log.error("JSON Stream exception: " + jx.toString());
			jx.printStackTrace();
		} catch( JAXBException je ) {
			log.error("JSON generation exception: " + je.toString());
			je.printStackTrace();
		}
	}

	protected static void load(SkosManager skosManager, SchemeFactoryType sourceType, ConceptScheme aScheme, Properties properties, String configFile, boolean reload) {
		// Transform an XML input into a standard format using an STX program
		// with "joost"
		// If output of JOOST can be JAX2 events, create directly the
		// ConceptScheme when answering to these events
		// This output will have to be modeled as a list of concepts which can be
		// repeated because additional information about an entry can be
		// specified later
		// ConceptScheme.xsd will have to be completed.
		// The list itself has a code (normally equal to the file name)
		// For each entry, the following fields are necessary: code, aliases to
		// this code, code of the different parents, names in different
		// languages, synonyms in different languages, codes along different
		// sub-authorities
		boolean caching = !reload;
		String cacheFileName = properties.getProperty("cache");
		if (cacheFileName == null || cacheFileName.length() == 0) {
			cacheFileName = configFile.replaceFirst("\\.[a-zA-Z0-9]*$", "")+"-cache.xml";
		}
		String refreshingHours = properties.getProperty("refresh"); // the refresh delay (hours) must be defined if URL
		int refresh = Integer.MAX_VALUE;
		if (refreshingHours != null && !refreshingHours.isEmpty() ) {
			try {
				refresh = Integer.valueOf(refreshingHours);
			} catch (NumberFormatException nfe) {
				log.error("Cache refreshing delay must ge given as a number of hours",nfe);
				refresh = Integer.MAX_VALUE;
			}
		}
		if (refresh <= 0) {  // A zero refresh delay stops caching 
			caching = false;
		}
		boolean isURL = true;
		String anURL = properties.getProperty("url");
		String sourceFileName = null;
		File sourceFile = null;
		if (anURL == null || anURL.length()==0) {
			isURL = false;
			sourceFileName = properties.getProperty("file"); // XML, TAB
			if (sourceFileName == null || sourceFileName.length() == 0) {
				sourceFileName = configFile.replaceFirst("\\.[a-zA-Z0-9]*$", "") + "." +sourceType.toString().toLowerCase();
			}
			sourceFile = new File(sourceFileName);
		}
		String aXSLT = properties.getProperty("xslt");
		if (aXSLT == null && sourceType != SchemeFactoryType.XML) {
			aXSLT = /*skosManager.getConfigDirectory()+*/sourceType.toString().toLowerCase()+"2skosXml.xslt";
		}
		if (aXSLT == null || aXSLT.isEmpty()) {
			caching = false;
		}

		NoDTDInputStream dataInputStream = null;
		String dataInputLocation = "";
		if (caching) {
			try {
				File cacheFile = new File(cacheFileName);
				if (cacheFile.length() == 0) {
					log.info(cacheFileName+" is empty, will be refreshed");
				} else {
					long cacheLastWritten = cacheFile.lastModified();
					long ageHours = (new Date().getTime() - cacheLastWritten) / 3600000L;
					if (  ageHours >= refresh ) {
						caching = false;
						log.info(cacheFileName+" too old, will be refreshed");
					} else {
						if (sourceFile != null && cacheLastWritten < sourceFile.lastModified()) {
							caching = false;
							log.info(cacheFileName+" older than "+sourceFileName+", will be refreshed");
						} else {
							dataInputLocation = cacheFile.getAbsolutePath();
							dataInputStream = new NoDTDInputStream(new FileInputStream(cacheFile));
						}
					}
				}
			} catch( IOException ioe ) {
				dataInputStream = null;
			} 
		}
		if (!caching || dataInputStream == null) {
			caching = false;
			try {
				if (isURL) {
					URL sourceURL = new URL(anURL);
					dataInputLocation = anURL;
					dataInputStream = new NoDTDInputStream(sourceURL.openStream());
				} else {
					dataInputLocation = sourceFileName;
					//File confFile = new File(configFile);
					//completeFileName = confFile.getCanonicalFile()+sourceFileName+".xml";
					dataInputStream = new NoDTDInputStream(new FileInputStream(sourceFileName));
				}
			} catch( IOException ioe ) {
				log.error("Error opening stream to read=" +(isURL?anURL:sourceFileName), ioe);
				ioe.printStackTrace();
			} 
		}
		if (dataInputStream != null) {
			JAXBContext jc = null;
			Unmarshaller u = null;
			try {
				jc = getJC();
				u = jc.createUnmarshaller();
			} catch( JAXBException je ) {
				je.printStackTrace();
			}
			if (jc != null) {
				ConceptScheme xScheme = null;	
				if (!caching && aXSLT != null && !aXSLT.isEmpty()) {
					// Generate a Transformer.
					Transformer transformer = null;
					//XMLInputFactory inputFactory = XMLInputFactory.newInstance();
					try
					{
						transformer = XMLUtil.openStandardTransformer(skosManager.getConfigDirectory(), aXSLT);
						//transformer = XMLUtil.openStreamingTransformer(inputFactory,aXSLT,aScheme.getAbout());
					}
					catch (Exception e)
					{
						log.error("XSLT Compilation problem with " + aXSLT,e);
						transformer = null;
					}
					if (transformer == null) return; // No XSLT: nothing possible!
					// Give access to Scheme configuration parameters within the XSLT
					for (String key : properties.stringPropertyNames())
					{
						String value = (String) properties.get(key);
						transformer.setParameter(key, value);
					}
					// http://rezarahim.blogspot.com/2010/05/chunking-out-big-xml-with-stax-and-jaxb.html :
					/*
import static javax.xml.stream.XMLStreamConstants.CHARACTERS;
import static javax.xml.stream.XMLStreamConstants.START_ELEMENT;

import java.io.FileOutputStream;
import java.io.FileReader;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamReader;
import javax.xml.stream.XMLStreamWriter;

public class ChunkReadWrite {

 public static void main(String[] args) throws Exception {

  String inFile = "/tmp/contacts.xml";
  String outFile = "/tmp/contactSt.xml";

  // set up a StAX reader
  XMLInputFactory xmlif = XMLInputFactory.newInstance();
  XMLStreamReader xmlr = xmlif.createXMLStreamReader(new FileReader(inFile));

  //set up JAXB context
  JAXBContext jaxbContext = JAXBContext.newInstance(Contact.class);
  Unmarshaller um = jaxbContext.createUnmarshaller();

  //Set up out put file. 
  FileOutputStream outputStream = new FileOutputStream( outFile );  
  XMLOutputFactory outputFactory=XMLOutputFactory.newInstance();
  XMLStreamWriter xmlStreamWriter= outputFactory.createXMLStreamWriter(outputStream,"UTF-8");

  //If we want to indent StAX out put  
  //com.sun.xml.internal.txw2.output.IndentingXMLStreamWriter sw = new IndentingXMLStreamWriter(defaultWriter);
        //sw.setIndentStep("    ");

  // Set up JAXB for Contact class
  Marshaller marshaller = JAXBContext.newInstance( Contact.class ).createMarshaller();
     marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
     marshaller.setProperty("jaxb.fragment", Boolean.TRUE);

  // move to the root element and check its name.
  xmlr.nextTag();
  xmlr.require(START_ELEMENT, null, "addressBook");

  //Pre-fill the file
  xmlStreamWriter.writeStartDocument("UTF-8","1.0");
  xmlStreamWriter.writeComment(" A test for StAX ");
  xmlStreamWriter.writeStartElement("addressBook");
  outputStream.flush();

  // move to the first contact element.
  xmlr.nextTag(); 
  while (xmlr.getEventType() == START_ELEMENT) {

      xmlr.require(START_ELEMENT, null, "contact");
  // unmarshall one contact element into a JAXB Contact object
   Contact contact = (Contact) um.unmarshal(xmlr);

   //Do any processing ....

   //write out the contact object 
      marshaller.marshal(contact, xmlStreamWriter);

   //marshaller.marshal(contact, System.out);

      // skip the whitespace between contacts.
   if (xmlr.getEventType() == CHARACTERS) {
    xmlr.next(); 
   }
  }

  xmlStreamWriter.writeEndElement();
  outputStream.flush();
  outputStream.close();
 }
}

---BINDED CLASS---
import javax.xml.bind.annotation.XmlRootElement;

@XmlRootElement
public class Contact {

 private String name;
 private String email;

 public Contact(){

 }

 public String getName() {
  return name;
 }

 public void setName(String name) {
  this.name = name;
 }

 public String getEmail() {
  return email;
 }

 public void setEmail(String email) {
  this.email = email;
 }


}

					 */
					// set up a StAX reader
					XMLInputFactory xmlif = XMLInputFactory.newInstance();
					XMLStreamReader xmlr =null;
					try {
						xmlr = xmlif.createXMLStreamReader(dataInputStream);
					} catch (XMLStreamException e1) {
						log.error(dataInputStream, e1);
						RunCommand.closeAble(dataInputStream, dataInputLocation);
						return;
					}
					String chunking = properties.getProperty("chunk");

					FileOutputStream resultStream = null;
					XMLStreamWriter xmlStreamWriter= null;
					try
					{
						//resultWriter = new OutputStreamWriter(resultStream,"UTF-8");
						//StreamResult transformResult = new StreamResult(resultWriter);
						XMLOutputFactory outputFactory=XMLOutputFactory.newInstance();

						if (chunking == null || chunking.isEmpty()) {
							// Returns resultWriter to free mem before finishing with stringReader...
							// Perform the transformation.
							//transformer.transform(input, transformResult);
							StAXSource input = new StAXSource(xmlr);
							//Set up output file. 
							resultStream = new FileOutputStream(cacheFileName);
							xmlStreamWriter = outputFactory.createXMLStreamWriter(resultStream,"UTF-8");
							StAXResult xmlResult = new StAXResult(xmlStreamWriter);
							transformer.transform(input, xmlResult);
						} else {
							//Set up output file. 
							resultStream = new NoXmlDeclarationFileOutputStream(cacheFileName);
							//Pre-fill the file
							//<?xml version='1.0' encoding='UTF-8'?>
							xmlStreamWriter = outputFactory.createXMLStreamWriter(resultStream,"UTF-8");
							xmlStreamWriter.writeStartDocument("UTF-8","1.0");
							xmlStreamWriter.writeStartElement("conceptScheme");
							xmlStreamWriter.writeAttribute("about",aScheme.getAbout());
							xmlStreamWriter.writeComment("Cached XML serialization for scheme "+aScheme.getAbout());
							//<conceptScheme about="http://aims.fao.org/aos/agrovoc/agrovocScheme"><title lang="cs">...</title><concept about="3">
							xmlStreamWriter.flush();
							//							resultStream.flush();
							transformer.setOutputProperty(javax.xml.transform.OutputKeys.OMIT_XML_DECLARATION,"yes");
							((NoXmlDeclarationFileOutputStream)resultStream).setRemoveXmlDeclaration(true);
							while (xmlr.hasNext()) {
								try {
									// move to the first chunk.
									while (xmlr.hasNext()) {
										if (xmlr.isStartElement()) break;
										xmlr.next();
									}
									if (!xmlr.hasNext()) break;
									//log.error(xmlr.getName().toString());
									if (!chunking.equals(xmlr.getPrefix()+':'+xmlr.getLocalName())) {
										xmlr.next();
										continue;
									}
									// Perform the transformation.
									StAXSource input = new StAXSource(xmlr);
									XMLStreamWriter insideStreamWriter= outputFactory.createXMLStreamWriter(resultStream,"UTF-8");
									StAXResult xmlResult = new StAXResult(insideStreamWriter);
									try {
										transformer.transform(input, xmlResult);
										insideStreamWriter.flush();
										insideStreamWriter.close();
									}
									catch (XMLStreamException ioe) {
										log.error(aScheme.getAbout() + ioe);										
									}
									catch (javax.xml.transform.TransformerException ioe) {
										log.error(aScheme.getAbout() + ioe);										
									}
								} catch( Exception ioe ) {
									log.error(aScheme.getAbout(), ioe);
									break;
								}
							}
/*
							File inputFile = new File(cacheFileName+".bug");
							FileInputStream inputStream = new FileInputStream(inputFile);
							resultStream = new FileOutputStream(cacheFileName);
							byte[] buffer = new byte[4096];
							//TODO: BUG quand ligne de 4093 caracteres!!!
							int nb;
							nb = inputStream.read(buffer,0,80); // Skip first <?xml...
							if (nb > 0) {
								resultStream.write(buffer,0,nb);
								while ( (nb = inputStream.read(buffer) ) > 0) {
									int ps = Util.searchBuffer(buffer,nb,XMLUtil.begXmlDecl,0);
									while (ps >= 0) {
										if (ps > (nb-80)) {
											resultStream.write(buffer, 0, ps);
											for (int i=ps; i < nb; i++) {
												buffer[i-ps] = buffer[i];
											}
											nb = nb - ps;
											ps = 0;
											int got = inputStream.read(buffer,nb,buffer.length-nb);
											if (got > 0) nb = nb + got;
										}
										int es = Util.searchBuffer(buffer,nb,XMLUtil.endXmlDecl,ps);
										if (es < 0) {
											ps = ps+XMLUtil.begXmlDecl.length;
											es=ps-XMLUtil.endXmlDecl.length;
										}
										resultStream.write(buffer, 0, ps);
										resultStream.write('\r');
										resultStream.write('\n');
										int j = 0;
										for (int i=es+XMLUtil.endXmlDecl.length; i < nb; i++) {
											buffer[j] = buffer[i];
											j++;
										}
										nb = j;
										if ((buffer.length-nb) > 80) {
											int got = inputStream.read(buffer,nb,buffer.length-nb);
											if (got > 0) nb = nb + got;
										}
										ps = Util.searchBuffer(buffer,nb,XMLUtil.begXmlDecl,0);
									}
									resultStream.write(buffer, 0, nb);
								}
							}
							inputStream.close();
							inputFile.delete();
*/
						}
						// Write document end. This closes all open structures
						xmlStreamWriter.writeEndDocument();
						// Close the writer to flush the output
						xmlStreamWriter.close();
						resultStream.close();
					}
					catch (TransformerException e)
					{
						log.error("Transformer exception: " + e.toString());
						e.printStackTrace();
					}
					catch (XMLStreamException e)
					{
						log.error("XMLStream exception: " + e.toString());
						e.printStackTrace();
					}
					catch (FileNotFoundException e)
					{
						log.error("File not found exception: " + e.toString());
					}
					catch (IOException e)
					{
						log.error("IO exception: " + e.toString());
						e.printStackTrace();
					}
					finally {
						try {
							if (xmlStreamWriter != null) xmlStreamWriter.close();
						} catch( XMLStreamException ioe ) {
							log.error("Cache file for write=" +cacheFileName, ioe);
						}
					}
					StreamSource source = null;
					try {
						dataInputStream = new NoDTDInputStream(new FileInputStream(cacheFileName));
						source = new StreamSource(dataInputStream);
						try {
							xScheme = (ConceptScheme)u.unmarshal( source );
						} catch( JAXBException je ) {
							log.error("XML Parsing exception: " + je.toString());
							xScheme = null;
							je.printStackTrace();
						}
					} catch( IOException ioe ) {
						log.error("Error reading "+cacheFileName, ioe);
					} finally {
						RunCommand.closeAble(dataInputStream, "cached data read stream " +cacheFileName);
					}
				} else {
					try {
						// Loading "title","description","namespace","uri","help","icon","display","create","internalNote","notation","concept"
						xScheme = (ConceptScheme)u.unmarshal( dataInputStream );
					} catch( JAXBException je ) {
						log.error("XML Parsing exception: " + je.toString());
						xScheme = null;
						je.printStackTrace();
					}
				}
				if (xScheme != null) {
					readConceptAddendum( skosManager, xScheme, sourceType, configFile, properties, caching, refresh, jc, u);
				}
				if (xScheme != null) {
					loadNewScheme(aScheme,xScheme,reload);
				}
			}
			RunCommand.closeAble(dataInputStream, "data input stream " +dataInputLocation);
		}
	}

	static void loadNewScheme (ConceptScheme aScheme, ConceptScheme xScheme, boolean reload) {
		if ( ! aScheme.getAbout().equals(xScheme.getAbout())) {
			SchemeFactory.log.warn("Loading XML scheme "+aScheme.getAbout()+", within XML, about is different:"+xScheme.getAbout());
		}
		// This is necessary because concepts in xScheme are not created by JAXB in the context of the aScheme
		String schemeApplication = xScheme.getApplication();
		if (schemeApplication != null && schemeApplication.length() > 0) {
			aScheme.setApplication(schemeApplication);
		}
		TermList titles = xScheme.getTitle();
		if (titles != null && titles.size() > 0) { 
			aScheme.setTitle(titles);
		}
		TermList descriptions = xScheme.getDescription();
		if (descriptions != null && descriptions.size() > 0) { 
			aScheme.setDescription(descriptions);
		}
		String namespace = xScheme.getNamespace();
		if (namespace != null && namespace.length() > 0) {
			aScheme.setNamespace(namespace);
		}
		String uri = xScheme.getUri();
		if (uri != null && uri.length() > 0) {
			aScheme.setUri(uri);
		}
		String schemaOrg = xScheme.getSchemaOrg();
		if (schemaOrg != null && schemaOrg.length() > 0) {
			aScheme.setSchemaOrg(schemaOrg);
		}
		LinkedList<String> supraLangs = xScheme.getSupraLanguage();
		if (supraLangs != null && supraLangs.size() > 0) { 
			SchemeFactory.log.debug(xScheme.getAbout()+" unmarshalled, supraLanguages="+supraLangs);
			aScheme.setSupraLanguage(supraLangs);
		}
		UrlList urls = xScheme.getHelp();
		if (urls != null && urls.size() > 0) { 
			aScheme.setHelp(urls);
		}
		urls = xScheme.getIcon();
		if (urls != null && urls.size() > 0) { 
			aScheme.setIcon(urls);
		}
		urls = xScheme.getDisplay();
		if (urls != null && urls.size() > 0) { 
			aScheme.setDisplay(urls);
		}
		urls = xScheme.getCreate();
		if (urls != null && urls.size() > 0) { 
			aScheme.setCreate(urls);
		}
		String internalNote = xScheme.getInternalNote();
		if (internalNote != null && internalNote.length() > 0) {
			aScheme.setInternalNote(internalNote);
		}
		LinkedList<NotationScheme> notations = xScheme.getNotation();
		if (notations != null) {
			for (NotationScheme aNotation : notations) {
				aNotation.setPrimary(aScheme);
				SkosManager aManagerInstance = aScheme.getManagerInstance();
				aNotation.setManagerInstance(aManagerInstance);
				aScheme.putNotationScheme(aNotation);
				aManagerInstance.putScheme(aNotation); // So it is remembered!!!
			}
		}
		SchemeUsageList usages = xScheme.getUsage();
		if (usages != null) {
			aScheme.setUsage(usages);
		}

		boolean cleaned = false;
		xScheme.setConcept(ConceptScheme.mergeDown(xScheme.getConcept()));
		//for (Concept aConcept : xScheme.getConcept()) {
			
		//}
		if (reload) {
			aScheme.reloadConcept(xScheme.getConcept());
		}
		else {
			aScheme.setConcept(xScheme.getConcept());
			if (aScheme.getConcept() != null) {
				for (Concept aConcept: aScheme.getConcept()) {
					aConcept.setUsage(aConcept.getUsage());
				}
			}
		}
	}

	static void readConceptAddendum (SkosManager skosManager, ConceptScheme xScheme, SchemeFactoryType sourceType,
			String configFile, Properties properties, boolean cachingAllowed, int refresh,
			JAXBContext jc, Unmarshaller u) {
		String cacheFileName = properties.getProperty("cache");
		if (cacheFileName == null || cacheFileName.length() == 0) {
			cacheFileName = configFile.replaceFirst("\\.[a-zA-Z0-9]*$", "")+ File.separator + "[about]-cache.xml";
		}
		String anURL = properties.getProperty("concept.url");
		if (anURL != null && !anURL.isEmpty()) {
			String gentleness = properties.getProperty("concept.ms");
			long gentle = 0;
			if (gentleness != null && !gentleness.isEmpty()) {
				try {
					gentle = Long.valueOf(gentleness);
				} catch (NumberFormatException e) {
					log.error("concept.ms="+gentleness+" is not a number of milliseconds?");
				}
			}
			String aXSLT = properties.getProperty("concept.xslt");
			Transformer transformer = null;
			if (aXSLT == null && sourceType != SchemeFactoryType.XML) {
				aXSLT = skosManager.getConfigDirectory()+"concept-"+sourceType.toString().toLowerCase()+"2skosXml.xslt";
			}
			if (aXSLT != null && !aXSLT.isEmpty()) {
				// Generate a Transformer.
				//XMLInputFactory inputFactory = XMLInputFactory.newInstance();
				try
				{
					transformer = XMLUtil.openStandardTransformer(skosManager.getConfigDirectory(), aXSLT);
					//transformer = XMLUtil.openStreamingTransformer(inputFactory,aXSLT,aScheme.getAbout());
				}
				catch (Exception e)
				{
					log.error("Concept XSLT Program Compile exception: " + e.toString());
					transformer = null;
				}
				if (transformer == null) return; // No XSLT: nothing possible!
				// Give access to Scheme configuration parameters within the XSLT
				for (String key : properties.stringPropertyNames())
				{
					String value = (String) properties.get(key);
					transformer.setParameter(key, value);
				}
				transformer.setParameter("scheme", xScheme.getAbout());
			}
			for (Concept aConcept : xScheme.getConcept()) {
				boolean caching = cachingAllowed;
				InputStream dataInputStream = null;
				String dataInputLocation = "";
				String conceptCacheName = Util.replaceAllNoRegExp(cacheFileName,"[about]",aConcept.getAbout());
				if (caching) {
					try {
						File cacheFile = new File(conceptCacheName);
						if (!cacheFile.exists()) {
							caching = false;
						} else {
							long cacheLastWritten = cacheFile.lastModified();
							long ageHours = (new Date().getTime() - cacheLastWritten) / 3600000L;
							if (  ageHours >= refresh ) {
								caching = false;
								log.info(ageHours+"hours: "+conceptCacheName+" too old, will be refreshed ("+cacheLastWritten+")");
							} else {
								if (cacheFile.length() == 0) {
									log.info(conceptCacheName+" is empty, SKIPPED, not refreshed");
									continue;
								} else {
									log.info(conceptCacheName+" will be reused as a cache copy");
									dataInputLocation = cacheFile.getAbsolutePath();
									dataInputStream = new NoDTDInputStream(new FileInputStream(cacheFile));
								}
							}
						}
					} catch( IOException ioe ) {
						log.info(conceptCacheName+" not accessible, will be created", ioe);
						dataInputStream = null;
					} 
				}
				if (!caching || dataInputStream == null) {
					caching = false;
					String aConceptURL = Util.replaceAllNoRegExp(anURL,"[about]",aConcept.getAbout());
					try {
						URL sourceURL = new URL(aConceptURL);
						dataInputLocation = aConceptURL;
						// be gentle enough to sleep a bit!
						if (gentle > 0) {
							try {
								Thread.sleep(gentle);
							} catch (InterruptedException e) {
								log.error("Sleeping "+gentle+"ms was interrupted.",e);
							}
						}
						dataInputStream = new NoDTDInputStream(sourceURL.openStream());
					} catch( IOException ioe ) {
						log.error("Error opening stream to read=" + aConceptURL, ioe);
						ioe.printStackTrace();
					} 
				}
				if (dataInputStream != null) {
					Concept xConcept = null;	
					if (transformer != null && !caching) {
						transformer.setParameter("about", aConcept.getAbout());
						// set up a StAX reader
						XMLInputFactory xmlif = XMLInputFactory.newInstance();
						XMLStreamReader xmlr =null;
						try {
							xmlr = xmlif.createXMLStreamReader(dataInputStream);
						} catch (XMLStreamException e1) {
							log.error(dataInputStream, e1);
							RunCommand.closeAble(dataInputStream, dataInputLocation);
							return;
						}

						FileReader fileReader = null;
						FileOutputStream resultStream = null;
						XMLStreamWriter xmlStreamWriter= null;
						try
						{
							//resultWriter = new OutputStreamWriter(resultStream,"UTF-8");
							//StreamResult transformResult = new StreamResult(resultWriter);
							XMLOutputFactory outputFactory=XMLOutputFactory.newInstance();

							// Returns resultWriter to free mem before finishing with stringReader...
							// Perform the transformation.
							//transformer.transform(input, transformResult);
							StAXSource input = new StAXSource(xmlr);
							//Set up output file. 
							resultStream = new FileOutputStream(conceptCacheName);
							xmlStreamWriter = outputFactory.createXMLStreamWriter(resultStream,"UTF-8");
							StAXResult xmlResult = new StAXResult(xmlStreamWriter);
							transformer.transform(input, xmlResult);

							// Write document end. This closes all open structures
							xmlStreamWriter.writeEndDocument();
							// Close the writer to flush the output
							xmlStreamWriter.close();
							resultStream.close();
						}
						catch (TransformerException e)
						{
							log.error("Transformer exception: " + e.toString());
							e.printStackTrace();
						}
						catch (XMLStreamException e)
						{
							log.error("XMLStream exception: " + e.toString());
							e.printStackTrace();
						}
						catch (FileNotFoundException e)
						{
							log.error("File not found exception: " + e.toString());
						}
						catch (IOException e)
						{
							log.error("IO exception: " + e.toString());
							e.printStackTrace();
						}
						finally {
							try {
								if (xmlStreamWriter != null) xmlStreamWriter.close();
							} catch( XMLStreamException ioe ) {
								log.error("Concept cache file for write=" +conceptCacheName, ioe);
							}
						}
						StreamSource source = null;
						try {
							fileReader = new FileReader(conceptCacheName);
							source = new StreamSource(fileReader);
							try {
								xConcept = (Concept)u.unmarshal( source );
							} catch( JAXBException je ) {
								log.error("Concept XML Parsing exception: " + je.toString());
								xConcept = null;
								je.printStackTrace();
							}
						} catch( IOException ioe ) {
							log.error("Error reading concept in "+conceptCacheName, ioe);
						} finally {
							RunCommand.closeAble(fileReader, "concept cached data read stream " +conceptCacheName);
						}
					} else {
						try {
							xConcept = (Concept)u.unmarshal( dataInputStream );
						} catch( JAXBException je ) {
							log.error("Concept XML Parsing exception: " + je.toString());
							xConcept = null;
							je.printStackTrace();
						}
					}
					if (xConcept != null) {
						aConcept.mergeImporting(xConcept);
					}
				}
			}
		}
	}
}
